
import pandas as pd
from gensim import corpora, models
from gensim.test.utils import common_corpus
from gensim.models import LdaSeqModel
import csv
import openpyxl as pxl
import jieba
import jieba.posseg as pseg
import re

stopwords = []
with open('thesaurus/stop_word.txt', 'r', encoding='utf-8') as f:
    for line in f:
        stopwords.append(line.strip())

# 加载自定义词典
jieba.load_userdict('thesaurus/user_word.txt')

# 读取csv文件
df = pd.read_csv('new.csv')
content = df['content']
content = content.apply(lambda x: re.sub(r'\d+\.\d+', '', str(x)))
df['content'] = content
df.to_csv('new_test.csv', index=False)

def preprocess(text):
    words = jieba.cut(text)
    words = [word for word in words if word not in stopwords and len(word) > 1]
    return words

df['result'] = df['content'].apply(preprocess)

# 输出结果到result列中
df.to_csv('result.csv', index=False)


##该函数doc2bow()只计算每个不同单词的出现次数，将单词转换为整数单词id，并将结果作为稀疏向量返回
dictionary = corpora.Dictionary(df['result'])
corpus = [dictionary.doc2bow(text) for text in df['result']]
time_slice = [20,20,20,20,20]   #设置这个语料库的间隔
num_topics = 10  #设置主题数，此处为10个主题

#建立DTM模型
ldaseq = LdaSeqModel(corpus=corpus, id2word=dictionary, time_slice=time_slice, num_topics=num_topics,)
ldaseq.num_terms = 20
#将语料库、词典、参数加载入模型中进行训练
print('输出指定时期主题分布，此处第一个时期主题分布')
corpusTopic = ldaseq.print_topics(time=0)  # 输出指定时期主题分布，此处第一个时期主题分布
print(ldaseq.print_topics(time=0))
print('='*50)
print('主题的在不同时期演变')
print(ldaseq.print_topic_times(topic=0))
print(ldaseq.print_topic_times(topic=1))


workbook3 = pxl.Workbook()
worksheet3 = workbook3.active

# 把主题动态结果写入xlsx
curRow = 1
for i in range(10):
    for row in ldaseq.print_topic_times(topic=i):
        if row:  # 去除空行
            worksheet3.cell(row = curRow, column= 1, value= "topic" + str(i))
            for J in range(len(row)):
                worksheet3.cell(row=curRow, column=(J + 1) * 2, value= str(row[J][0]))
                worksheet3.cell(row=curRow, column=(J + 1) * 2 + 1, value= str(row[J][1]))
            curRow += 1
workbook3.save("dynamic_theme.xlsx")


print('='*50)

workbook = pxl.Workbook()
worksheet = workbook.active
i = 0
for i in range(99):#excel条数 文档-主题分配 多少条数据就打印多少条
    row = ldaseq.doc_topics(i)
    row_list = row.tolist()
    worksheet.append(row_list)
#保存xlsx文件
workbook.save('doc_theme.xlsx')
# 不同主题不同时期的情况
ldaseq.save('dtm_news')
dtm_model = ldaseq.load('dtm_news')
